We first defined the working directory, load the required library and load the transaction dataset.
# Set Working Directory
setwd("~/SANDY/web_analytics/1_CA1/CA1_r11/3_Sequential_Rules_Analysis")
#Load Required Libraries
library(arulesViz)
library(dplyr)
library(datetime)
library(arulesSequences)
#Load Required data
dt1 = read.csv(file="train.csv")
The following is the first 5 rows of the dataset.
dt1[1:5,]
The following is the Basic Summary of the dataset.
summary(dt1)
session_id datetime_y item_id category
Min. : 11 9/14/2014 11:12:15: 6 Min. :214507331 Min. : 0.000
1st Qu.: 1319978 4/13/2014 18:02:47: 5 1st Qu.:214716973 1st Qu.: 0.000
Median : 8998744 8/31/2014 18:27:41: 5 Median :214839313 Median :14.000
Mean : 6867169 4/12/2014 16:43:56: 4 Mean :214786898 Mean : 7.634
3rd Qu.:10209851 4/13/2014 17:51:10: 4 3rd Qu.:214853154 3rd Qu.:14.000
Max. :11562121 4/13/2014 9:52:22 : 4 Max. :214990340 Max. :14.000
(Other) :346048
Recode Event Id to numeric ascending order.
df$eventID <- df$eventID[1] <- 1
for (i in 1:length(df$sequenceID)) {
if (i == 1) {df$eventID[i] <- 1} else
if( df$sequenceID[i-1] == df$sequenceID[i])
{
df$eventID[i] <-df$eventID[i-1]+1
}
}
df[1:5,]
df1 <- df
# Check dummy columns
df1$seq_test <- df1$sequenceID
df1$sequenceID <-df1$sequenceID[1] <- 1
df1[1:5,]
Recode Sequence Id to numeric ascending order.
for (i in 1:length(df1$seq_test)) {
if (i == 1) {df$sequenceID[i] <- 1} else
if(df1$seq_test[i-1] == df1$seq_test[i]){ df1$sequenceID[i] = df1$sequenceID[i-1] }
else {df1$sequenceID[i] = df1$sequenceID[i-1]+1}
}
df1[1:5,]
Export the data out as .txt files and re-construct the Transaction Basket file.
write.table(df2, "seq_format.txt", sep=" ", row.names = FALSE, col.names = FALSE, quote = FALSE)
data <- read_baskets(con = "seq_format.txt", info = c("sequenceID","eventID","SIZE"))
Show the Sequences Rules.
as(head(data), "data.frame")
Run CSpade Algorithm.
For CSAPDE algoritm you might set some lags so that you can extract rules from sequence of transactions with the lag.
We set the minimum support of rules to 0.5%.
seqs <- cspade(data, parameter = list(support = 0.0005), control = list(verbose = TRUE))
parameter specification:
support : 5e-04
maxsize : 10
maxlen : 10
algorithmic control:
bfstype : FALSE
verbose : TRUE
summary : FALSE
tidLists : FALSE
preprocessing ... 2 partition(s), 7.27 MB [0.48s]
mining transactions ... 0.02 MB [0.16s]
reading sequences ... [0.15s]
total elapsed time: 0.784s
View the Sequences.
as(seqs,"data.frame") # view the sequences
Convert extracted sequential rules to data frame and Filter rules with more than one sequence
scrul.dt <- as(seqs,"data.frame")
scrul.dt$sequence <- gsub("df3\\$cart2\\=|<|>","",scrul.dt$sequence)
scrul.dt1 <- scrul.dt[count.fields(textConnection(scrul.dt$sequence),sep = ",")>1,]
scrul.dt1
scrul.dt1[10,]
Each of unique sequences happened on the same date. For rule 10,If a customer’s first purchase is 214853102, his second purchase would be 214854840 which is frequent for around 2% of session user.
Induced the Sequences Rules.
seqrules <- ruleInduction(seqs, confidence = 0.5,control = list(verbose = TRUE))
processing ... 1271 itemsets, 876 rules [0.00s]
The following is the Sequence Rules with 50% Confidences.
as(seqrules,"data.frame") # view the rules
Testing Sequence Rules
We first defined the Working Functions and load the Test dataset
#remove duplicate items from a basket (itemstrg)
uniqueitems <- function(itemstrg) {
unique(as.list(strsplit(gsub(" ","",itemstrg),","))[[1]])
}
# execute ruleset using item as rule antecedent (handles single item antecedents only)
makepreds <- function(item, rulesDF) {
antecedent = paste("<{",item,"}> =>",sep="") # NOTE: diff from assoc analysis same fn
firingrules = rulesDF[grep(antecedent, rulesDF$rule,fixed=TRUE),1] # rules is now rule
#gsub(" ","",toString(sub(">}","",sub(".*=> <{","",firingrules))))
gsub(" ", "", toString(sub('\\}>', '', sub(".*=> <\\{", "", firingrules))))
}
# count how many predictions are in the basket of items already seen by that user
# Caution : refers to "baskets" as a global
checkpreds <- function(preds, baskID) {
plist = preds[[1]]
blist = baskets[baskets$basketID == baskID,"items"][[1]]
cnt = 0
for (p in plist) {
if (p %in% blist) cnt = cnt+1
}
cnt
}
# count all predictions made
countpreds <- function(predlist) {
len = length(predlist)
if (len > 0 && (predlist[[1]] == "")) 0 # avoid counting an empty list
else len
}
# Load the test data
testegs = read.csv(file="test.csv")
testegs = testegs[,c(1,3)]
colnames(testegs) <- c("basketID","items") # set standard names, in case they are different in the data file
# Display top 5 rows
testegs[1:5,]
#execute rules against test data
rulesDF = as(seqrules,"data.frame")
testegs$preds = apply(testegs,1,function(X) makepreds(X["items"], rulesDF))
# extract unique predictions for each test user
userpreds = as.data.frame(aggregate(preds ~ basketID, data = testegs, paste, collapse=","))
userpreds$preds = apply(userpreds,1,function(X) uniqueitems(X["preds"]))
# extract unique items for each test user
baskets = as.data.frame(aggregate(items ~ basketID, data = testegs, paste, collapse=","))
baskets$items = apply(baskets,1,function(X) uniqueitems(X["items"]))
#count how many unique predictions made are correct, i.e. have previously been bought (or rated highly) by the user
correctpreds = sum(apply(userpreds,1,function(X) checkpreds(X["preds"],X["basketID"])))
# count total number of unique predictions made
totalpreds = sum(apply(userpreds,1,function(X) countpreds(X["preds"][[1]])))
precision = correctpreds*100/totalpreds
cat("precision=", precision, "corr=",correctpreds,"total=",totalpreds)
precision= 93.81443 corr= 182 total= 194
LS0tCnRpdGxlOiAiUiBOT1RFQk9PSyAtICoqU2VxdWVudGlhbCBSdWxlcyBBbmFseXNpcyoqIgpvdXRwdXQ6IGh0bWxfbm90ZWJvb2sKLS0tCgpXZSBmaXJzdCBkZWZpbmVkIHRoZSB3b3JraW5nIGRpcmVjdG9yeSwgbG9hZCB0aGUgcmVxdWlyZWQgbGlicmFyeSBhbmQgbG9hZCB0aGUgdHJhbnNhY3Rpb24gZGF0YXNldC4gICAKYGBge3J9CiMgU2V0IFdvcmtpbmcgRGlyZWN0b3J5CnNldHdkKCJ+L1NBTkRZL3dlYl9hbmFseXRpY3MvMV9DQTEvQ0ExX3IxMS8zX1NlcXVlbnRpYWxfUnVsZXNfQW5hbHlzaXMiKQoKI0xvYWQgUmVxdWlyZWQgTGlicmFyaWVzCmxpYnJhcnkoYXJ1bGVzVml6KQpsaWJyYXJ5KGRwbHlyKQpsaWJyYXJ5KGRhdGV0aW1lKQpsaWJyYXJ5KGFydWxlc1NlcXVlbmNlcykKCiNMb2FkIFJlcXVpcmVkIGRhdGEgCmR0MSA9IHJlYWQuY3N2KGZpbGU9InRyYWluLmNzdiIpCgpgYGAKCiMjIyMgVGhlIGZvbGxvd2luZyBpcyB0aGUgZmlyc3QgKio1Kiogcm93cyBvZiB0aGUgZGF0YXNldC4KYGBge3J9CmR0MVsxOjUsXQpgYGAKCiMjIyMgVGhlIGZvbGxvd2luZyBpcyB0aGUgKipCYXNpYyBTdW1tYXJ5Kiogb2YgdGhlIGRhdGFzZXQuCmBgYHtyfQpzdW1tYXJ5KGR0MSkKYGBgCgojIyMjIFRyYW5zZm9ybSB0aGUgZGF0YXNldC4KYGBge3J9CmRmIDwtIGR0MSAlPiUKICAgICAgYXJyYW5nZShkYXRldGltZV95KSAlPiUKICAgICAgYXJyYW5nZShzZXNzaW9uX2lkKSAlPiUKICAgICAgdW5pcXVlKCkgJT4lCiAgICAgIGdyb3VwX2J5KHNlc3Npb25faWQpICU+JQogICAgICAjc3VtbWFyaXNlKGNhcnQ9cGFzdGUoaXRlbV9pZCxjb2xsYXBzZT0iOyIpKSAlPiUKICAgICAgdW5ncm91cCgpCmRmWzE6NSxdCmBgYApDcmVhdGUgYWRkaXRpb25hbCBjb2x1bW5zIGZvciB0cmFuc2Zvcm1hdGlvbi4gCmBgYHtyfQpkZiRzZXF1ZW5jZUlEIDwtIGRmJHNlc3Npb25faWQKZGYkZXZlbnRJRCAgICA8LSBkZiRzZXNzaW9uX2lkCmRmJFNJWkUgICAgICAgPC0gJzEnCmRmJGl0ZW1zICAgICAgPC0gZGYkaXRlbV9pZApkZiRpdGVtcyAgICAgIDwtIGFzLmZhY3RvcihkZiRpdGVtcykKZGZbMTo1LF0KYGBgCgojIyMjIFJlY29kZSAqKkV2ZW50IElkKiogdG8gbnVtZXJpYyBhc2NlbmRpbmcgb3JkZXIuCmBgYHtyfQpkZiRldmVudElEIDwtIGRmJGV2ZW50SURbMV0gPC0gMQpmb3IgKGkgaW4gMTpsZW5ndGgoZGYkc2VxdWVuY2VJRCkpIHsKICAKICBpZiAoaSA9PSAxKSB7ZGYkZXZlbnRJRFtpXSA8LSAxfSBlbHNlCiAgaWYoIGRmJHNlcXVlbmNlSURbaS0xXSA9PSBkZiRzZXF1ZW5jZUlEW2ldKQogIHsKICAgICAgZGYkZXZlbnRJRFtpXSA8LWRmJGV2ZW50SURbaS0xXSsxIAogICAgCiAgICB9Cn0KYGBgCmBgYHtyfQpkZlsxOjUsXQpgYGAKCmBgYHtyfQpkZjEgPC0gZGYKIyBDaGVjayBkdW1teSBjb2x1bW5zCmRmMSRzZXFfdGVzdCA8LSBkZjEkc2VxdWVuY2VJRApkZjEkc2VxdWVuY2VJRCA8LWRmMSRzZXF1ZW5jZUlEWzFdIDwtIDEKCmRmMVsxOjUsXQpgYGAKCiMjIyMgUmVjb2RlICoqU2VxdWVuY2UgSWQqKiB0byBudW1lcmljIGFzY2VuZGluZyBvcmRlci4KYGBge3J9CmZvciAoaSBpbiAxOmxlbmd0aChkZjEkc2VxX3Rlc3QpKSB7CiAgCiAgaWYgKGkgPT0gMSkge2RmJHNlcXVlbmNlSURbaV0gPC0gMX0gZWxzZSAKICBpZihkZjEkc2VxX3Rlc3RbaS0xXSA9PSBkZjEkc2VxX3Rlc3RbaV0peyBkZjEkc2VxdWVuY2VJRFtpXSA9IGRmMSRzZXF1ZW5jZUlEW2ktMV0gfQogIGVsc2Uge2RmMSRzZXF1ZW5jZUlEW2ldID0gZGYxJHNlcXVlbmNlSURbaS0xXSsxfQp9CmBgYApgYGB7cn0KZGYxWzE6NSxdCmBgYAoKIyMjIyBUaGUgKipmaW5hbCBzZXF1ZW5jZSBmb3JtYXQqKiBkYXRhIGlzIGFzIGZvbGxvd3M6LQpgYGB7cn0KZGYyICAgICAgICAgICA8LSBkZjFbYyg1LDYsNyw4KV0KZGYyJHNlcXVlbmNlSUQgPC0gYXMuaW50ZWdlcihkZjIkc2VxdWVuY2VJRCkKZGYyJGV2ZW50SUQgICA8LSBhcy5pbnRlZ2VyKGRmMiRldmVudElEKQpkZjIkU0laRSAgICAgIDwtIGFzLmludGVnZXIoZGYyJFNJWkUpCmRmMiA8LSBkZjJbb3JkZXIoZGYyJHNlcXVlbmNlSUQsZGYyJGV2ZW50SUQpLF0KI3NlcWNoa3B0MQpkZjJbMTo1LF0KYGBgCgojIyMjIEV4cG9ydCB0aGUgZGF0YSBvdXQgYXMgKioudHh0KiogZmlsZXMgYW5kIHJlLWNvbnN0cnVjdCB0aGUgKipUcmFuc2FjdGlvbiBCYXNrZXQqKiBmaWxlLgpgYGB7cn0Kd3JpdGUudGFibGUoZGYyLCAic2VxX2Zvcm1hdC50eHQiLCBzZXA9IiAiLCByb3cubmFtZXMgPSBGQUxTRSwgY29sLm5hbWVzID0gRkFMU0UsIHF1b3RlID0gRkFMU0UpCmRhdGEgPC0gcmVhZF9iYXNrZXRzKGNvbiA9ICJzZXFfZm9ybWF0LnR4dCIsIGluZm8gPSBjKCJzZXF1ZW5jZUlEIiwiZXZlbnRJRCIsIlNJWkUiKSkKYGBgCgojIyMjIFNob3cgKipUcmFuc2FjdGlvbiBPYmplY3QqKiBJbmZvcm1hdGlvbgpgYGB7cn0KIHRyYW5zYWN0aW9uSW5mbyhkYXRhKQpgYGAKCiMjIyMgU2hvdyB0aGUgKipTZXF1ZW5jZXMgUnVsZXMqKi4KYGBge3J9CmFzKGhlYWQoZGF0YSksICJkYXRhLmZyYW1lIikKYGBgCgojIyMjIFJ1biAqKkNTcGFkZSBBbGdvcml0aG0qKi4gICAgCkZvciBDU0FQREUgYWxnb3JpdG0geW91IG1pZ2h0IHNldCBzb21lIGxhZ3Mgc28gdGhhdCB5b3UgY2FuIGV4dHJhY3QgcnVsZXMgZnJvbSBzZXF1ZW5jZSBvZiB0cmFuc2FjdGlvbnMgd2l0aCB0aGUgbGFnLiAgIApXZSBzZXQgdGhlIG1pbmltdW0gc3VwcG9ydCBvZiBydWxlcyB0byAqKjAuNSUqKi4KYGBge3J9CnNlcXMgPC0gY3NwYWRlKGRhdGEsIHBhcmFtZXRlciA9IGxpc3Qoc3VwcG9ydCA9IDAuMDAwNSksIGNvbnRyb2wgPSBsaXN0KHZlcmJvc2UgPSBUUlVFKSkKYGBgCgojIyMjIFZpZXcgdGhlICoqU2VxdWVuY2VzKiouCmBgYHtyfQphcyhzZXFzLCJkYXRhLmZyYW1lIikgICMgdmlldyB0aGUgc2VxdWVuY2VzCmBgYAoKIyMjIyBDb252ZXJ0IGV4dHJhY3RlZCBzZXF1ZW50aWFsIHJ1bGVzIHRvIGRhdGEgZnJhbWUgYW5kIEZpbHRlciBydWxlcyB3aXRoIG1vcmUgdGhhbiBvbmUgc2VxdWVuY2UKYGBge3J9CnNjcnVsLmR0IDwtIGFzKHNlcXMsImRhdGEuZnJhbWUiKQpzY3J1bC5kdCRzZXF1ZW5jZSA8LSBnc3ViKCJkZjNcXCRjYXJ0MlxcPXw8fD4iLCIiLHNjcnVsLmR0JHNlcXVlbmNlKQoKc2NydWwuZHQxIDwtIHNjcnVsLmR0W2NvdW50LmZpZWxkcyh0ZXh0Q29ubmVjdGlvbihzY3J1bC5kdCRzZXF1ZW5jZSksc2VwID0gIiwiKT4xLF0Kc2NydWwuZHQxCmBgYAoKYGBge3J9CnNjcnVsLmR0MVsxMCxdCmBgYApFYWNoIG9mIHVuaXF1ZSBzZXF1ZW5jZXMgaGFwcGVuZWQgb24gdGhlIHNhbWUgZGF0ZS4gRm9yIHJ1bGUgMTAsSWYgYSBjdXN0b21lcuKAmXMgZmlyc3QgcHVyY2hhc2UgaXMgMjE0ODUzMTAyLCBoaXMgc2Vjb25kIHB1cmNoYXNlIHdvdWxkIGJlIDIxNDg1NDg0MCB3aGljaCBpcyBmcmVxdWVudCBmb3IgYXJvdW5kIDIlIG9mIHNlc3Npb24gdXNlci4KCiMjIyMgSW5kdWNlZCB0aGUgU2VxdWVuY2VzIFJ1bGVzLgpgYGB7cn0Kc2VxcnVsZXMgPC0gcnVsZUluZHVjdGlvbihzZXFzLCBjb25maWRlbmNlID0gMC41LGNvbnRyb2wgPSBsaXN0KHZlcmJvc2UgPSBUUlVFKSkKYGBgCgojIyMjIFRoZSBmb2xsb3dpbmcgaXMgdGhlICoqU2VxdWVuY2UgUnVsZXMgd2l0aCA1MCUgQ29uZmlkZW5jZXMqKi4KYGBge3J9CmFzKHNlcXJ1bGVzLCJkYXRhLmZyYW1lIikgICMgdmlldyB0aGUgcnVsZXMKYGBgCgojIyMgKipUZXN0aW5nIFNlcXVlbmNlIFJ1bGVzKiogICAgCldlIGZpcnN0IGRlZmluZWQgdGhlICoqV29ya2luZyBGdW5jdGlvbnMqKiBhbmQgbG9hZCB0aGUgKipUZXN0KiogZGF0YXNldApgYGB7cn0KI3JlbW92ZSBkdXBsaWNhdGUgaXRlbXMgZnJvbSBhIGJhc2tldCAoaXRlbXN0cmcpCnVuaXF1ZWl0ZW1zIDwtIGZ1bmN0aW9uKGl0ZW1zdHJnKSB7CiAgdW5pcXVlKGFzLmxpc3Qoc3Ryc3BsaXQoZ3N1YigiICIsIiIsaXRlbXN0cmcpLCIsIikpW1sxXV0pCn0KIyBleGVjdXRlIHJ1bGVzZXQgdXNpbmcgaXRlbSBhcyBydWxlIGFudGVjZWRlbnQgKGhhbmRsZXMgc2luZ2xlIGl0ZW0gYW50ZWNlZGVudHMgb25seSkKbWFrZXByZWRzIDwtIGZ1bmN0aW9uKGl0ZW0sIHJ1bGVzREYpIHsKICBhbnRlY2VkZW50ID0gcGFzdGUoIjx7IixpdGVtLCJ9PiA9PiIsc2VwPSIiKSAjIE5PVEU6IGRpZmYgZnJvbSBhc3NvYyBhbmFseXNpcyBzYW1lIGZuCiAgZmlyaW5ncnVsZXMgPSBydWxlc0RGW2dyZXAoYW50ZWNlZGVudCwgcnVsZXNERiRydWxlLGZpeGVkPVRSVUUpLDFdICMgcnVsZXMgaXMgbm93IHJ1bGUKICAjZ3N1YigiICIsIiIsdG9TdHJpbmcoc3ViKCI+fSIsIiIsc3ViKCIuKj0+IDx7IiwiIixmaXJpbmdydWxlcykpKSkKICBnc3ViKCIgIiwgIiIsIHRvU3RyaW5nKHN1YignXFx9PicsICcnLCBzdWIoIi4qPT4gPFxceyIsICIiLCBmaXJpbmdydWxlcykpKSkKfQojIGNvdW50IGhvdyBtYW55IHByZWRpY3Rpb25zIGFyZSBpbiB0aGUgYmFza2V0IG9mIGl0ZW1zIGFscmVhZHkgc2VlbiBieSB0aGF0IHVzZXIgCiMgQ2F1dGlvbiA6IHJlZmVycyB0byAiYmFza2V0cyIgYXMgYSBnbG9iYWwKY2hlY2twcmVkcyA8LSBmdW5jdGlvbihwcmVkcywgYmFza0lEKSB7CiAgcGxpc3QgPSBwcmVkc1tbMV1dCiAgYmxpc3QgPSBiYXNrZXRzW2Jhc2tldHMkYmFza2V0SUQgPT0gYmFza0lELCJpdGVtcyJdW1sxXV0KICBjbnQgPSAwIAogIGZvciAocCBpbiBwbGlzdCkgewogICAgaWYgKHAgJWluJSBibGlzdCkgY250ID0gY250KzEKICB9CiAgY250Cn0KIyBjb3VudCBhbGwgcHJlZGljdGlvbnMgbWFkZQpjb3VudHByZWRzIDwtIGZ1bmN0aW9uKHByZWRsaXN0KSB7CiAgbGVuID0gbGVuZ3RoKHByZWRsaXN0KQogIGlmIChsZW4gPiAwICYmIChwcmVkbGlzdFtbMV1dID09ICIiKSkgMCAjIGF2b2lkIGNvdW50aW5nIGFuIGVtcHR5IGxpc3QKICBlbHNlIGxlbgp9CiMgTG9hZCB0aGUgdGVzdCBkYXRhCnRlc3RlZ3MgPSByZWFkLmNzdihmaWxlPSJ0ZXN0LmNzdiIpCnRlc3RlZ3MgPSB0ZXN0ZWdzWyxjKDEsMyldCmNvbG5hbWVzKHRlc3RlZ3MpIDwtIGMoImJhc2tldElEIiwiaXRlbXMiKSAgIyBzZXQgc3RhbmRhcmQgbmFtZXMsIGluIGNhc2UgdGhleSBhcmUgZGlmZmVyZW50IGluIHRoZSBkYXRhIGZpbGUKIyBEaXNwbGF5IHRvcCA1IHJvd3MKdGVzdGVnc1sxOjUsXQpgYGAKCmBgYHtyfQojZXhlY3V0ZSBydWxlcyBhZ2FpbnN0IHRlc3QgZGF0YQpydWxlc0RGID0gYXMoc2VxcnVsZXMsImRhdGEuZnJhbWUiKQp0ZXN0ZWdzJHByZWRzID0gYXBwbHkodGVzdGVncywxLGZ1bmN0aW9uKFgpIG1ha2VwcmVkcyhYWyJpdGVtcyJdLCBydWxlc0RGKSkKCiMgZXh0cmFjdCB1bmlxdWUgcHJlZGljdGlvbnMgZm9yIGVhY2ggdGVzdCB1c2VyCnVzZXJwcmVkcyA9IGFzLmRhdGEuZnJhbWUoYWdncmVnYXRlKHByZWRzIH4gYmFza2V0SUQsIGRhdGEgPSB0ZXN0ZWdzLCBwYXN0ZSwgY29sbGFwc2U9IiwiKSkKdXNlcnByZWRzJHByZWRzID0gYXBwbHkodXNlcnByZWRzLDEsZnVuY3Rpb24oWCkgdW5pcXVlaXRlbXMoWFsicHJlZHMiXSkpCgojIGV4dHJhY3QgdW5pcXVlIGl0ZW1zIGZvciBlYWNoIHRlc3QgdXNlcgpiYXNrZXRzID0gYXMuZGF0YS5mcmFtZShhZ2dyZWdhdGUoaXRlbXMgfiBiYXNrZXRJRCwgZGF0YSA9IHRlc3RlZ3MsIHBhc3RlLCBjb2xsYXBzZT0iLCIpKQpiYXNrZXRzJGl0ZW1zID0gYXBwbHkoYmFza2V0cywxLGZ1bmN0aW9uKFgpIHVuaXF1ZWl0ZW1zKFhbIml0ZW1zIl0pKQoKI2NvdW50IGhvdyBtYW55IHVuaXF1ZSBwcmVkaWN0aW9ucyBtYWRlIGFyZSBjb3JyZWN0LCBpLmUuIGhhdmUgcHJldmlvdXNseSBiZWVuIGJvdWdodCAob3IgcmF0ZWQgaGlnaGx5KSBieSB0aGUgdXNlcgpjb3JyZWN0cHJlZHMgPSBzdW0oYXBwbHkodXNlcnByZWRzLDEsZnVuY3Rpb24oWCkgY2hlY2twcmVkcyhYWyJwcmVkcyJdLFhbImJhc2tldElEIl0pKSkKCiMgY291bnQgdG90YWwgbnVtYmVyIG9mIHVuaXF1ZSBwcmVkaWN0aW9ucyBtYWRlCnRvdGFscHJlZHMgPSBzdW0oYXBwbHkodXNlcnByZWRzLDEsZnVuY3Rpb24oWCkgY291bnRwcmVkcyhYWyJwcmVkcyJdW1sxXV0pKSkgCnByZWNpc2lvbiA9IGNvcnJlY3RwcmVkcyoxMDAvdG90YWxwcmVkcwpjYXQoInByZWNpc2lvbj0iLCBwcmVjaXNpb24sICJjb3JyPSIsY29ycmVjdHByZWRzLCJ0b3RhbD0iLHRvdGFscHJlZHMpCmBgYAo=